# -*- coding: utf-8 -*-
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import time, datetime
import commonlib, re
from MongoModel import MongoModel
import traceback
import html2text
import workerpool
from MongoDbLog import MongoDbLog
from BeautifulSoup import BeautifulSoup

MONGO_SERVER = 'beta.mana.vn'
MONGO_PORT = 27017
DOWNLOAD_PATH = '/home/hoangnamhai/HarvestedData/tintuc/tym/bongda/'
PREFIX = '/uploads/bongda/'
LOG = commonlib.getMongoLog(MONGO_SERVER, MONGO_PORT, 'bd7mcn')

class Bd7m(MongoModel):
    
    __listGiaiDau = {'92':'Anh', '34':'Ý', '39':'Đức', '93':'Pháp', '85':'Tây Ban Nha', '810': 'Việt Nam'}
    
    def __init__(self, host='localhost', port=27017):
        MongoModel.__init__(self, host, port)
    
    def standardizeDatetimeOfTeamNPlayer(self, timeStr):
        if re.search(r"\d+-\d+-\d+", timeStr):
            return re.sub(r"(\d{4})-(\d{1,2})-(\d{1,2})", lambda m: "{0}/{1}/{2}".format(m.group(3), m.group(2), m.group(1)), timeStr)
#            return datetime.datetime.strptime(timeStr, "%Y-%m-%d").strftime("%d/%m/%Y")
        if re.search(r"(\d{4})-(\d{1,2})", timeStr):
            return re.sub(r"(\d{4})-(\d{1,2})", lambda m: "{0}/{1}".format(m.group(2), m.group(1)), timeStr)
        return timeStr
    
    def getTeamList(self, maGiaidau):
        LOG.debug('start getTeamList: %s' % self.__listGiaiDau[maGiaidau])
        url = 'http://data.7m.cn/matches_data/{0}/vn/standing.js?a={1}'.format(maGiaidau, time.time())
        data = []
        try:
            html = commonlib.getHTML(url)
            f_sds_ti = ''
            for line in re.findall(r'.*var\s(.+);', html):
                exec(line)
            for i in range(len(f_sds_ti)):
                data.append(f_sds_ti[i])
        except:
            LOG.error(traceback.format_exc())
        return data
    
    def playerPosition(self, posId):
        data = {'0': 'Tiền đạo',
            '1': 'Tiền vệ',
            '2': 'Hậu vệ',
            '3': 'Thủ môn'
            }
        if commonlib.dataTypeName(posId) != 'str': posId = str(posId)
        return data[posId]
    
    def playerInfo(self, playerId, teamId):
        LOG.debug('start playerInfo: {0}, {1}'.format(playerId, teamId))
        teamUrl = 'http://data2.7m.cn/player_data/{0}/vn/index.shtml'.format(teamId)
        url = 'http://data2.7m.cn/sdt.aspx?t=2&id={id}&e=3&f=0'.format(id=playerId)
        try:
            html = commonlib.getHTML(url, headers={'Referer': teamUrl})
            p_name, birthday, stature, avoirdupois, join_date, shirt_number, country = '', '', '', '', '', '', ''
            photos, club, p_club, o_club, note = [], '', '', '', ''
            for line in re.findall(r'.*var\s(.+);', html):
                try:
                    exec(line)
                except:
                    pass
            playerPhotos = self.saveImage(photos, 'http://data.7m.cn')
            playerImg = commonlib.downloadImage("http://data.7m.cn/player_data/{0}/logo_img/player_photo.jpg".format(playerId), DOWNLOAD_PATH, PREFIX) 
            note = commonlib.convertHTMLEntitiesToUnicode(note)
            note = re.sub(r'<br />+', '\n', note)
            note = re.sub(r'<.+?>', '', note)
            data = {'_id': playerId,
                'name': commonlib.encodeUTF8Str(p_name),
                'birthday': self.standardizeDatetimeOfTeamNPlayer(birthday),
                'height': stature,
                'weight': avoirdupois,
                'joinDate': self.standardizeDatetimeOfTeamNPlayer(join_date),
                'shirtNumber': shirt_number,
                'country': country,
                'clubName': club,
                'clubId': teamId,
                'previousClub': p_club,
                'otherClub': o_club,
                'note': note,
                'photos': playerPhotos,
                'avatar': playerImg
                }
            db = self.connection['bongda']
            collection = db['players']
            collection.save(data)
            print('-----------------------------')
            print('Name: %s' % data['name'])
            print('Shirt Number: %s' % data['shirtNumber'])
            print('-----------------------------')
        except:
            LOG.error(traceback.format_exc())
        
    def teamInfo(self, teamId):
        LOG.debug('start teamInfo: {0}'.format(teamId))
        url = 'http://data.7m.cn/team_data/{0}/vn/dt.js'.format(teamId)
        try:
            html = commonlib.getHTML(url)
            team_name, team_edate, coach_pre_club, coach_his_club, team_country, team_city = '', '', '', '', '', ''
            team_capacity, team_addr, team_website, team_avgAge, team_resume, team_stadium = '', '', '', '', '', ''
            coach_bh, coach_name, coach_birthday, coach_club, coach_join, coach_height, coach_gj = '', '', '', '', '', '', ''
            plist_num, plist_i, plist_n, plist_pos, team_photo = [], [], [], [], []
            team_logo = 1
            for line in re.findall(r'.*var\s(.+);', html):
                try:
                    exec(line)
                except:
                    pass
            print '---------------------------------'
            print 'Team: ', commonlib.encodeUTF8Str(team_name)
            
            photos = self.saveImage(team_photo, 'http://data.7m.cn')
            players = []
            for i in range(len(plist_i)):
                players.append({'_id': plist_i[i], 'name': plist_n[i], 'shirtNumber': plist_num[i], 'position': self.playerPosition(plist_pos[i])})
                self.playerInfo(plist_i[i], teamId)
            if team_logo == 1:
                linkLogo = "http://data.7m.cn/team_data/{0}/logo_Img/club_logo.jpg"
            elif team_logo == 2:
                linkLogo = "http://data.7m.cn/team_data/{0}/logo_Img/club_logo.gif"
            else:
                linkLogo = "http://data.7m.cn/team_data/share_Img/err.gif"
            logoUrl = commonlib.downloadImage(linkLogo.format(teamId), DOWNLOAD_PATH, PREFIX)
            team_resume = commonlib.convertHTMLEntitiesToUnicode(team_resume)
            team_resume = re.sub(r"<br />+", '\n', team_resume)
            team_resume = re.sub(r'<.+?>', '', team_resume)
            data = {'_id': teamId,
                'lastupdate': datetime.datetime.now(),
                'timestamp': time.time(),
                'name': commonlib.encodeUTF8Str(team_name),
                'establish': self.standardizeDatetimeOfTeamNPlayer(commonlib.encodeUTF8Str(team_edate)),
                'country': commonlib.encodeUTF8Str(team_country),
                'city': commonlib.encodeUTF8Str(team_city),
                'stadium': commonlib.encodeUTF8Str(team_stadium),
                'capacity': team_capacity,
                'address': commonlib.encodeUTF8Str(team_addr),
                'website': team_website,
                'avgAge': team_avgAge,
                'resume': team_resume,
                'coachId': coach_bh,
                'coachName': commonlib.encodeUTF8Str(coach_name),
                'coachBirthday': self.standardizeDatetimeOfTeamNPlayer(coach_birthday),
                'coachClub': coach_club,
                'coachJoin': coach_join,
                'coachPreClub': coach_pre_club,
                'coachHistoryClub': coach_his_club,
                'coachHeight': coach_height,
                'coachGj': commonlib.encodeUTF8Str(coach_gj),
                'photos': photos,
                'players': players,
                'logo': logoUrl
                }
            print photos
            db = self.connection['bongda']
            collection = db['teams']
            collection.save(data)            
        except:
            LOG.error(traceback.format_exc())
    
    def saveImage(self, arr, rootUrl):
        if commonlib.dataTypeName(arr) != 'list': return []
        data = []
        for item in arr:
            src = commonlib.extractWithRegEx(r'.+src="(.+)"\swidth', item, 1)
            if src != '': 
                src = commonlib.urlJoin(rootUrl, src)
                if src != '':
                    dlink = commonlib.downloadImage(src, DOWNLOAD_PATH, PREFIX)
                    if dlink != '': data.append(dlink)
        return data
            
    def process(self):
        try:
            for i in self.__listGiaiDau.keys():
                data = self.getTeamList(i)
                pool = workerpool.WorkerPool(size=5)
                pool.map(self.teamInfo, data)
                pool.shutdown()
                pool.wait()
        except:
            LOG.critical(traceback.format_exc())

if __name__ == '__main__':
    LOG.info('start crawler teams data.7m.cn')
    bd = Bd7m(MONGO_SERVER, MONGO_PORT)
    bd.teamInfo(444)
#    bd.process()
#    bd.playerInfo(15095, 239)
    LOG.info('finished crawler teams data.7m.cn at %s' % str(datetime.datetime.now()))
    os._exit(1)
    
